library(ggplot2)
library(dplyr)
library(tidyr)
library("ggpubr")
library(LDATS)
library(stringr)
source("utils.R")
Visualization on CIFAR100. We are using data of three neural networks trained on reduced CIFAR100 training set. Half of the CIFAR100 training set was extracted as a validation set. We then divided both the reduced training set and validation set into 5 disjoint subsets and trained an ensemble on each of them. This was done in 10 replications, each time with random split of the training set into validation and new training set. In this visualization, we are trying to inspect the outputs deeper, mainly to make sense of strange behavior of nll metric for ensemble outputs.
base_dir <- "../data/data_train_val_half_c100"
repls <- 0:9
folds <- 0:4
classes <- 100
nets_outputs <- load_network_outputs(base_dir, repls)
ens_outputs <- load_ensemble_outputs(base_dir, repls, folds)
net_results <- read.csv(file.path(base_dir, "net_accuracies.csv"))
ens_results <- read.csv(file.path(base_dir, "ensemble_accuracies.csv"))
preds <- nets_outputs$test_outputs
for (ri in repls + 1)
{
for (net_i in seq_along(nets_outputs[["networks"]]))
{
preds[ri, net_i, ,] <- softmax(preds[ri, net_i, , ])
}
}
nets_test_cor_probs <- gather(preds, 1 + nets_outputs$test_labels[1, ], 3, 4)
nets_test_cor_probs <- melt(nets_test_cor_probs)
nets_test_cor_probs <- nets_test_cor_probs[, c(-3, -4)]
names(nets_test_cor_probs) <- c("replication", "network", "prediction")
nets_test_cor_probs$network <- as.factor(nets_test_cor_probs$network)
levels(nets_test_cor_probs$network) <- nets_outputs$networks
nets_cor_preds_histo <- ggplot(data=nets_test_cor_probs) + geom_histogram(mapping=aes(x=prediction), binwidth=0.01) + facet_wrap(~network) + scale_y_log10()
nets_cor_preds_histo

val_ens_cor_probs <- gather(ens_outputs$val_training, 1 + nets_outputs$test_labels[1, ], 4, 5)
val_ens_cor_probs <- melt(val_ens_cor_probs)
val_ens_cor_probs <- val_ens_cor_probs[, c(-4, -5)]
names(val_ens_cor_probs) <- c("replication", "method", "fold", "prediction")
val_ens_cor_probs$method <- as.factor(val_ens_cor_probs$method)
levels(val_ens_cor_probs$method) <- ens_outputs$methods
val_ens_cor_preds_histo <- ggplot(data=val_ens_cor_probs) + geom_histogram(mapping=aes(x=prediction), binwidth=0.01) + facet_wrap(~method) + scale_y_log10() + ggtitle("Probabilities predicted for the correct class - ens trained on val")
val_ens_cor_preds_histo

val_ens_zero_counts <- ggplot(data=val_ens_cor_probs[val_ens_cor_probs$prediction <= 0, ]) + geom_histogram(mapping=aes(x=method), stat="count")
Warning: Ignoring unknown parameters: binwidth, bins, pad
val_ens_zero_counts

val_ens_nll <- ggplot(data=ens_results) + geom_boxplot(mapping=aes(x=method, y=nll)) + facet_wrap(~train_set)
val_ens_nll

train_ens_cor_probs <- gather(ens_outputs$train_training, 1 + nets_outputs$test_labels[1, ], 4, 5)
train_ens_cor_probs <- melt(train_ens_cor_probs)
train_ens_cor_probs <- train_ens_cor_probs[, c(-4, -5)]
names(train_ens_cor_probs) <- c("replication", "method", "fold", "prediction")
train_ens_cor_probs$method <- as.factor(train_ens_cor_probs$method)
levels(train_ens_cor_probs$method) <- ens_outputs$methods
train_ens_cor_preds_histo <- ggplot(data=train_ens_cor_probs) + geom_histogram(mapping=aes(x=prediction), binwidth=0.01) + facet_wrap(~method) + scale_y_log10() + ggtitle("Probabilities predicted for the correct class - ens trained on train")
train_ens_cor_preds_histo
Warning: Transformation introduced infinite values in continuous y-axis
Warning: Removed 64 rows containing missing values (geom_bar).

train_ens_zero_counts <- ggplot(data=train_ens_cor_probs[train_ens_cor_probs$prediction <= 0, ]) + geom_histogram(mapping=aes(x=method), stat="count")
Warning: Ignoring unknown parameters: binwidth, bins, pad
train_ens_zero_counts

val_ens_cor_probs$train_type <- "vt"
train_ens_cor_probs$train_type <- "tt"
ens_cor_probs <- rbind(val_ens_cor_probs, train_ens_cor_probs)
ens_cor_preds_histo <- ggplot(data=ens_cor_probs) + geom_histogram(mapping=aes(x=prediction), binwidth=0.01) + facet_grid(rows=vars(method), cols=vars(train_type)) + scale_y_log10() + ggtitle("Probabilities predicted for the correct class")
ens_cor_preds_histo
Warning: Transformation introduced infinite values in continuous y-axis
Warning: Removed 64 rows containing missing values (geom_bar).

Very strange behavior for the bc method trained on training set. Needs further attention.
val_aggr_Rs <- np$load(file.path(base_dir, "val_training_class_aggr_R.npy"))
train_aggr_Rs <- np$load(file.path(base_dir, "train_training_class_aggr_R.npy"))
df_val_aggr_Rs <- melt(val_aggr_Rs)
names(df_val_aggr_Rs) <- c("precision", "class", "class1", "class2", "prob")
df_train_aggr_Rs <- melt(train_aggr_Rs)
names(df_train_aggr_Rs) <- c("precision", "class", "class1", "class2", "prob")
df_val_aggr_Rs$train_type <- "val_training"
df_train_aggr_Rs$train_type <- "train_training"
df_aggr_Rs <- rbind(df_val_aggr_Rs, df_train_aggr_Rs)
df_aggr_Rs[, c("class", "class1", "class2")] <- lapply(df_aggr_Rs[, c("class", "class1", "class2")], as.factor)
df_aggr_Rs_diff <- df_aggr_Rs %>% pivot_wider(names_from = train_type, values_from = prob) %>% mutate(val_min_train = val_training - train_training)
for (cls in 1:classes)
{
cur_class_Rs <- df_aggr_Rs %>% filter(class == cls)
plot_cls <- ggplot(cur_class_Rs, aes(x = class2, y = class1)) +
geom_raster(aes(fill=prob)) +
facet_wrap(~train_type) +
scale_fill_gradient(low="grey90", high="red") +
scale_y_discrete(limits=rev, breaks=seq(0, classes, 10)) +
scale_x_discrete(breaks=seq(0, classes, 10)) +
labs(x="class 2", y="class 1", title=paste("Pairwise probabilities - class ", cls)) +
theme_bw()
print(plot_cls)
}




































































































Difference between these two LDA training methodologies are not well visible, so we will plot just the differences.
for (cls in 1:classes)
{
cur_class_Rs <- df_aggr_Rs_diff %>% filter(class == cls)
plot_cls <- ggplot(cur_class_Rs, aes(x = class2, y = class1)) +
geom_raster(aes(fill=val_min_train)) +
scale_fill_gradient2(low="blue", high="red", mid="white", midpoint=0) +
scale_y_discrete(limits=rev, breaks=seq(0, classes, 10)) +
scale_x_discrete(breaks=seq(0, classes, 10)) +
labs(x="class 2", y="class 1", title=paste("Pairwise probabilities differences - class ", cls)) +
theme_bw()
print(plot_cls)
}




































































































lda_coefs <- load_lda_coefs(base_dir, repls, folds)
for (cl1 in 1:19)
{
for (cl2 in (cl1 + 1):20)
{
cur_plt <- lda_coefs %>% filter(class1 == cl1 & class2 == cl2) %>% ggplot() + geom_boxplot(aes(x=coefficient, y=value)) +
facet_wrap(~train_type) + ggtitle(paste("Coefficients for class", cl1, "vs", cl2))
print(cur_plt)
}
}






























































































































































































Coefficients of LDA trained on validation set have lower variance and more similar values across the networks.
avg_lda_coefs <- lda_coefs %>% filter(coefficient != "interc") %>% group_by(class1, class2, precision, train_type, coefficient) %>% summarise( value = mean(value)) %>% ungroup()
`summarise()` has grouped output by 'class1', 'class2', 'precision', 'train_type'. You can override using the `.groups` argument.
avg_lda_coefs_vt <- avg_lda_coefs %>% filter(train_type=="val_training")
avg_lda_coefs_tt <- avg_lda_coefs %>% filter(train_type=="train_training")
avg_lda_coefs_vt$value <- avg_lda_coefs_vt$value - min(avg_lda_coefs_vt$value)
avg_lda_coefs_vt$value <- avg_lda_coefs_vt$value / max(avg_lda_coefs_vt$value)
avg_lda_coefs_tt$value <- avg_lda_coefs_tt$value - min(avg_lda_coefs_tt$value)
avg_lda_coefs_tt$value <- avg_lda_coefs_tt$value / max(avg_lda_coefs_tt$value)
avg_lda_coefs <- rbind(avg_lda_coefs_vt, avg_lda_coefs_tt)
avg_lda_c_w <- pivot_wider(avg_lda_coefs, names_from = coefficient, values_from = value)
avg_lda_c_w[, c("class1", "class2")] <- lapply(avg_lda_c_w[, c("class1", "class2")], as.factor)
raster_plot <- ggplot(avg_lda_c_w) +
geom_tile(aes(x=class2, y=class1, fill=rgb(densenet121, resnet34, xception))) +
scale_y_discrete(limits=rev, breaks=seq(0,classes, 10)) + scale_x_discrete(breaks=seq(0,classes, 10)) + scale_fill_identity() + facet_wrap(~train_type)
raster_plot

For validation set trained LDAs, red - densenet seems to be dominant. On the other hand for train set trained LDAs blue - xception has higher values.
avg_lda_coefs <- lda_coefs %>% filter(coefficient != "interc") %>% group_by(class1, class2, precision, train_type, coefficient) %>% summarise( value = mean(value)) %>% ungroup()
`summarise()` has grouped output by 'class1', 'class2', 'precision', 'train_type'. You can override using the `.groups` argument.
avg_lda_coefs_vt <- avg_lda_coefs %>% filter(train_type=="val_training")
avg_lda_coefs_tt <- avg_lda_coefs %>% filter(train_type=="train_training")
avg_lda_coefs_vt$value <- avg_lda_coefs_vt$value - min(avg_lda_coefs_vt$value)
avg_lda_coefs_vt$value <- avg_lda_coefs_vt$value / max(avg_lda_coefs_vt$value)
avg_lda_coefs_tt$value <- avg_lda_coefs_tt$value - min(avg_lda_coefs_tt$value)
avg_lda_coefs_tt$value <- avg_lda_coefs_tt$value / max(avg_lda_coefs_tt$value)
avg_lda_coefs <- rbind(avg_lda_coefs_vt, avg_lda_coefs_tt)
avg_lda_c_w <- pivot_wider(avg_lda_coefs, names_from = coefficient, values_from = value)
avg_lda_c_w[, c("class1", "class2")] <- lapply(avg_lda_c_w[, c("class1", "class2")], as.factor)
avg_lda_c_w$top_net <- factor(c("densenet121", "resnet34", "xception")[max.col(as.matrix(avg_lda_c_w[, c("densenet121", "resnet34", "xception")]))])
raster_plot <- ggplot(avg_lda_c_w) +
geom_tile(aes(x=class2, y=class1, fill=rgb(densenet121, resnet34, xception))) +
scale_y_discrete(limits=rev, breaks=seq(0,classes, 10)) + scale_x_discrete(breaks=seq(0,classes, 10)) + scale_fill_identity() + facet_wrap(~train_type)
raster_plot

coefs_grid <- ggplot(avg_lda_c_w, aes(x=class2, y=class1, fill=top_net)) +
geom_raster() +
scale_fill_brewer(type="qual") +
facet_wrap(~train_type) +
scale_y_discrete(breaks=seq(0, classes, 10), limits=rev) +
scale_x_discrete(breaks=seq(0, classes, 10)) +
guides(fill=guide_legend(title="Network")) +
xlab("Class") +
ylab("Class") +
ggtitle("Network with highest lda weight for class pairs") +
theme(plot.title = element_text(hjust = 0.5),
axis.ticks = element_blank(),
panel.grid.major = element_blank(),
panel.grid.minor = element_blank())
coefs_grid

LDAs trained on nn train set seems to be dominated by xception. LDAs trained on validation set by densenet.
LS0tDQp0aXRsZTogIk91dHB1dHMgaW5zcGVjdGlvbiBoYWxmIENJRkFSMTAwIg0Kb3V0cHV0Og0KICBwZGZfZG9jdW1lbnQ6IGRlZmF1bHQNCiAgaHRtbF9ub3RlYm9vazogZGVmYXVsdA0KLS0tDQoNCmBgYHtyfQ0KbGlicmFyeShnZ3Bsb3QyKQ0KbGlicmFyeShkcGx5cikNCmxpYnJhcnkodGlkeXIpDQpsaWJyYXJ5KCJnZ3B1YnIiKQ0KbGlicmFyeShMREFUUykNCmxpYnJhcnkoc3RyaW5ncikNCg0Kc291cmNlKCJ1dGlscy5SIikNCmBgYA0KDQpWaXN1YWxpemF0aW9uIG9uIENJRkFSMTAwLg0KV2UgYXJlIHVzaW5nIGRhdGEgb2YgdGhyZWUgbmV1cmFsIG5ldHdvcmtzIHRyYWluZWQgb24gcmVkdWNlZCBDSUZBUjEwMCB0cmFpbmluZyBzZXQuIEhhbGYgb2YgdGhlIENJRkFSMTAwIHRyYWluaW5nIHNldCB3YXMgZXh0cmFjdGVkIGFzIGEgdmFsaWRhdGlvbiBzZXQuIFdlIHRoZW4gZGl2aWRlZCBib3RoIHRoZSByZWR1Y2VkIHRyYWluaW5nIHNldCBhbmQgdmFsaWRhdGlvbiBzZXQgaW50byA1IGRpc2pvaW50IHN1YnNldHMgYW5kIHRyYWluZWQgYW4gZW5zZW1ibGUgb24gZWFjaCBvZiB0aGVtLiBUaGlzIHdhcyBkb25lIGluIDEwIHJlcGxpY2F0aW9ucywgZWFjaCB0aW1lIHdpdGggcmFuZG9tIHNwbGl0IG9mIHRoZSB0cmFpbmluZyBzZXQgaW50byB2YWxpZGF0aW9uIGFuZCBuZXcgdHJhaW5pbmcgc2V0Lg0KSW4gdGhpcyB2aXN1YWxpemF0aW9uLCB3ZSBhcmUgdHJ5aW5nIHRvIGluc3BlY3QgdGhlIG91dHB1dHMgZGVlcGVyLCBtYWlubHkgdG8gbWFrZSBzZW5zZSBvZiBzdHJhbmdlIGJlaGF2aW9yIG9mIG5sbCBtZXRyaWMgZm9yIGVuc2VtYmxlIG91dHB1dHMuDQoNCmBgYHtyfQ0KYmFzZV9kaXIgPC0gIi4uL2RhdGEvZGF0YV90cmFpbl92YWxfaGFsZl9jMTAwIg0KcmVwbHMgPC0gMDo5DQpmb2xkcyA8LSAwOjQNCmNsYXNzZXMgPC0gMTAwDQoNCm5ldHNfb3V0cHV0cyA8LSBsb2FkX25ldHdvcmtfb3V0cHV0cyhiYXNlX2RpciwgcmVwbHMpDQplbnNfb3V0cHV0cyA8LSBsb2FkX2Vuc2VtYmxlX291dHB1dHMoYmFzZV9kaXIsIHJlcGxzLCBmb2xkcykNCm5ldF9yZXN1bHRzIDwtIHJlYWQuY3N2KGZpbGUucGF0aChiYXNlX2RpciwgIm5ldF9hY2N1cmFjaWVzLmNzdiIpKQ0KZW5zX3Jlc3VsdHMgPC0gcmVhZC5jc3YoZmlsZS5wYXRoKGJhc2VfZGlyLCAiZW5zZW1ibGVfYWNjdXJhY2llcy5jc3YiKSkNCmBgYA0KDQoNCmBgYHtyfQ0KcHJlZHMgPC0gbmV0c19vdXRwdXRzJHRlc3Rfb3V0cHV0cw0KZm9yIChyaSBpbiByZXBscyArIDEpDQp7DQogIGZvciAobmV0X2kgaW4gc2VxX2Fsb25nKG5ldHNfb3V0cHV0c1tbIm5ldHdvcmtzIl1dKSkNCiAgew0KICAgIHByZWRzW3JpLCBuZXRfaSwgLF0gPC0gc29mdG1heChwcmVkc1tyaSwgbmV0X2ksICwgXSkNCiAgfQ0KfQ0KbmV0c190ZXN0X2Nvcl9wcm9icyA8LSBnYXRoZXIocHJlZHMsIDEgKyBuZXRzX291dHB1dHMkdGVzdF9sYWJlbHNbMSwgXSwgMywgNCkNCm5ldHNfdGVzdF9jb3JfcHJvYnMgPC0gbWVsdChuZXRzX3Rlc3RfY29yX3Byb2JzKQ0KbmV0c190ZXN0X2Nvcl9wcm9icyA8LSBuZXRzX3Rlc3RfY29yX3Byb2JzWywgYygtMywgLTQpXQ0KbmFtZXMobmV0c190ZXN0X2Nvcl9wcm9icykgPC0gYygicmVwbGljYXRpb24iLCAibmV0d29yayIsICJwcmVkaWN0aW9uIikNCm5ldHNfdGVzdF9jb3JfcHJvYnMkbmV0d29yayA8LSBhcy5mYWN0b3IobmV0c190ZXN0X2Nvcl9wcm9icyRuZXR3b3JrKQ0KbGV2ZWxzKG5ldHNfdGVzdF9jb3JfcHJvYnMkbmV0d29yaykgPC0gbmV0c19vdXRwdXRzJG5ldHdvcmtzDQpgYGANCg0KYGBge3J9DQpuZXRzX2Nvcl9wcmVkc19oaXN0byA8LSBnZ3Bsb3QoZGF0YT1uZXRzX3Rlc3RfY29yX3Byb2JzKSArIGdlb21faGlzdG9ncmFtKG1hcHBpbmc9YWVzKHg9cHJlZGljdGlvbiksIGJpbndpZHRoPTAuMDEpICsgZmFjZXRfd3JhcCh+bmV0d29yaykgKyBzY2FsZV95X2xvZzEwKCkNCm5ldHNfY29yX3ByZWRzX2hpc3RvDQpgYGANCg0KDQpgYGB7cn0NCnZhbF9lbnNfY29yX3Byb2JzIDwtIGdhdGhlcihlbnNfb3V0cHV0cyR2YWxfdHJhaW5pbmcsIDEgKyBuZXRzX291dHB1dHMkdGVzdF9sYWJlbHNbMSwgXSwgNCwgNSkNCnZhbF9lbnNfY29yX3Byb2JzIDwtIG1lbHQodmFsX2Vuc19jb3JfcHJvYnMpDQp2YWxfZW5zX2Nvcl9wcm9icyA8LSB2YWxfZW5zX2Nvcl9wcm9ic1ssIGMoLTQsIC01KV0NCm5hbWVzKHZhbF9lbnNfY29yX3Byb2JzKSA8LSBjKCJyZXBsaWNhdGlvbiIsICJtZXRob2QiLCAiZm9sZCIsICJwcmVkaWN0aW9uIikNCnZhbF9lbnNfY29yX3Byb2JzJG1ldGhvZCA8LSBhcy5mYWN0b3IodmFsX2Vuc19jb3JfcHJvYnMkbWV0aG9kKQ0KbGV2ZWxzKHZhbF9lbnNfY29yX3Byb2JzJG1ldGhvZCkgPC0gZW5zX291dHB1dHMkbWV0aG9kcw0KYGBgDQoNCmBgYHtyfQ0KdmFsX2Vuc19jb3JfcHJlZHNfaGlzdG8gPC0gZ2dwbG90KGRhdGE9dmFsX2Vuc19jb3JfcHJvYnMpICsgZ2VvbV9oaXN0b2dyYW0obWFwcGluZz1hZXMoeD1wcmVkaWN0aW9uKSwgYmlud2lkdGg9MC4wMSkgKyBmYWNldF93cmFwKH5tZXRob2QpICsgc2NhbGVfeV9sb2cxMCgpICsgZ2d0aXRsZSgiUHJvYmFiaWxpdGllcyBwcmVkaWN0ZWQgZm9yIHRoZSBjb3JyZWN0IGNsYXNzIC0gZW5zIHRyYWluZWQgb24gdmFsIikNCnZhbF9lbnNfY29yX3ByZWRzX2hpc3RvDQpgYGANCg0KYGBge3J9DQp2YWxfZW5zX3plcm9fY291bnRzIDwtIGdncGxvdChkYXRhPXZhbF9lbnNfY29yX3Byb2JzW3ZhbF9lbnNfY29yX3Byb2JzJHByZWRpY3Rpb24gPD0gMCwgXSkgKyBnZW9tX2hpc3RvZ3JhbShtYXBwaW5nPWFlcyh4PW1ldGhvZCksIHN0YXQ9ImNvdW50IikNCnZhbF9lbnNfemVyb19jb3VudHMNCmBgYA0KDQpgYGB7cn0NCnZhbF9lbnNfbmxsIDwtIGdncGxvdChkYXRhPWVuc19yZXN1bHRzKSArIGdlb21fYm94cGxvdChtYXBwaW5nPWFlcyh4PW1ldGhvZCwgeT1ubGwpKSArIGZhY2V0X3dyYXAofnRyYWluX3NldCkNCnZhbF9lbnNfbmxsDQpgYGANCg0KDQpgYGB7cn0NCnRyYWluX2Vuc19jb3JfcHJvYnMgPC0gZ2F0aGVyKGVuc19vdXRwdXRzJHRyYWluX3RyYWluaW5nLCAxICsgbmV0c19vdXRwdXRzJHRlc3RfbGFiZWxzWzEsIF0sIDQsIDUpDQp0cmFpbl9lbnNfY29yX3Byb2JzIDwtIG1lbHQodHJhaW5fZW5zX2Nvcl9wcm9icykNCnRyYWluX2Vuc19jb3JfcHJvYnMgPC0gdHJhaW5fZW5zX2Nvcl9wcm9ic1ssIGMoLTQsIC01KV0NCm5hbWVzKHRyYWluX2Vuc19jb3JfcHJvYnMpIDwtIGMoInJlcGxpY2F0aW9uIiwgIm1ldGhvZCIsICJmb2xkIiwgInByZWRpY3Rpb24iKQ0KdHJhaW5fZW5zX2Nvcl9wcm9icyRtZXRob2QgPC0gYXMuZmFjdG9yKHRyYWluX2Vuc19jb3JfcHJvYnMkbWV0aG9kKQ0KbGV2ZWxzKHRyYWluX2Vuc19jb3JfcHJvYnMkbWV0aG9kKSA8LSBlbnNfb3V0cHV0cyRtZXRob2RzDQpgYGANCg0KDQpgYGB7cn0NCnRyYWluX2Vuc19jb3JfcHJlZHNfaGlzdG8gPC0gZ2dwbG90KGRhdGE9dHJhaW5fZW5zX2Nvcl9wcm9icykgKyBnZW9tX2hpc3RvZ3JhbShtYXBwaW5nPWFlcyh4PXByZWRpY3Rpb24pLCBiaW53aWR0aD0wLjAxKSArIGZhY2V0X3dyYXAofm1ldGhvZCkgKyBzY2FsZV95X2xvZzEwKCkgKyBnZ3RpdGxlKCJQcm9iYWJpbGl0aWVzIHByZWRpY3RlZCBmb3IgdGhlIGNvcnJlY3QgY2xhc3MgLSBlbnMgdHJhaW5lZCBvbiB0cmFpbiIpDQp0cmFpbl9lbnNfY29yX3ByZWRzX2hpc3RvDQpgYGANCg0KDQpgYGB7cn0NCnRyYWluX2Vuc196ZXJvX2NvdW50cyA8LSBnZ3Bsb3QoZGF0YT10cmFpbl9lbnNfY29yX3Byb2JzW3RyYWluX2Vuc19jb3JfcHJvYnMkcHJlZGljdGlvbiA8PSAwLCBdKSArIGdlb21faGlzdG9ncmFtKG1hcHBpbmc9YWVzKHg9bWV0aG9kKSwgc3RhdD0iY291bnQiKQ0KdHJhaW5fZW5zX3plcm9fY291bnRzDQpgYGANCg0KYGBge3J9DQp2YWxfZW5zX2Nvcl9wcm9icyR0cmFpbl90eXBlIDwtICJ2dCINCnRyYWluX2Vuc19jb3JfcHJvYnMkdHJhaW5fdHlwZSA8LSAidHQiDQplbnNfY29yX3Byb2JzIDwtIHJiaW5kKHZhbF9lbnNfY29yX3Byb2JzLCB0cmFpbl9lbnNfY29yX3Byb2JzKQ0KYGBgDQoNCmBgYHtyfQ0KZW5zX2Nvcl9wcmVkc19oaXN0byA8LSBnZ3Bsb3QoZGF0YT1lbnNfY29yX3Byb2JzKSArIGdlb21faGlzdG9ncmFtKG1hcHBpbmc9YWVzKHg9cHJlZGljdGlvbiksIGJpbndpZHRoPTAuMDEpICsgZmFjZXRfZ3JpZChyb3dzPXZhcnMobWV0aG9kKSwgY29scz12YXJzKHRyYWluX3R5cGUpKSArIHNjYWxlX3lfbG9nMTAoKSArIGdndGl0bGUoIlByb2JhYmlsaXRpZXMgcHJlZGljdGVkIGZvciB0aGUgY29ycmVjdCBjbGFzcyIpDQplbnNfY29yX3ByZWRzX2hpc3RvDQpgYGANClZlcnkgc3RyYW5nZSBiZWhhdmlvciBmb3IgdGhlIGJjIG1ldGhvZCB0cmFpbmVkIG9uIHRyYWluaW5nIHNldC4gTmVlZHMgZnVydGhlciBhdHRlbnRpb24uDQoNCmBgYHtyfQ0KdmFsX2FnZ3JfUnMgPC0gbnAkbG9hZChmaWxlLnBhdGgoYmFzZV9kaXIsICJ2YWxfdHJhaW5pbmdfY2xhc3NfYWdncl9SLm5weSIpKQ0KdHJhaW5fYWdncl9ScyA8LSBucCRsb2FkKGZpbGUucGF0aChiYXNlX2RpciwgInRyYWluX3RyYWluaW5nX2NsYXNzX2FnZ3JfUi5ucHkiKSkNCmRmX3ZhbF9hZ2dyX1JzIDwtIG1lbHQodmFsX2FnZ3JfUnMpDQpuYW1lcyhkZl92YWxfYWdncl9ScykgPC0gYygicHJlY2lzaW9uIiwgImNsYXNzIiwgImNsYXNzMSIsICJjbGFzczIiLCAicHJvYiIpDQpkZl90cmFpbl9hZ2dyX1JzIDwtIG1lbHQodHJhaW5fYWdncl9ScykNCm5hbWVzKGRmX3RyYWluX2FnZ3JfUnMpIDwtIGMoInByZWNpc2lvbiIsICJjbGFzcyIsICJjbGFzczEiLCAiY2xhc3MyIiwgInByb2IiKQ0KZGZfdmFsX2FnZ3JfUnMkdHJhaW5fdHlwZSA8LSAidmFsX3RyYWluaW5nIg0KZGZfdHJhaW5fYWdncl9ScyR0cmFpbl90eXBlIDwtICJ0cmFpbl90cmFpbmluZyINCmRmX2FnZ3JfUnMgPC0gcmJpbmQoZGZfdmFsX2FnZ3JfUnMsIGRmX3RyYWluX2FnZ3JfUnMpDQpkZl9hZ2dyX1JzWywgYygiY2xhc3MiLCAiY2xhc3MxIiwgImNsYXNzMiIpXSA8LSBsYXBwbHkoZGZfYWdncl9Sc1ssIGMoImNsYXNzIiwgImNsYXNzMSIsICJjbGFzczIiKV0sIGFzLmZhY3RvcikNCg0KZGZfYWdncl9Sc19kaWZmIDwtIGRmX2FnZ3JfUnMgJT4lIHBpdm90X3dpZGVyKG5hbWVzX2Zyb20gPSB0cmFpbl90eXBlLCB2YWx1ZXNfZnJvbSA9IHByb2IpICU+JSBtdXRhdGUodmFsX21pbl90cmFpbiA9IHZhbF90cmFpbmluZyAtIHRyYWluX3RyYWluaW5nKQ0KYGBgDQoNCmBgYHtyfQ0KZm9yIChjbHMgaW4gMTpjbGFzc2VzKQ0Kew0KICBjdXJfY2xhc3NfUnMgPC0gZGZfYWdncl9ScyAlPiUgZmlsdGVyKGNsYXNzID09IGNscykNCiAgcGxvdF9jbHMgPC0gIGdncGxvdChjdXJfY2xhc3NfUnMsIGFlcyh4ID0gY2xhc3MyLCB5ID0gY2xhc3MxKSkgKyANCiAgICBnZW9tX3Jhc3RlcihhZXMoZmlsbD1wcm9iKSkgKyANCiAgICBmYWNldF93cmFwKH50cmFpbl90eXBlKSArDQogICAgc2NhbGVfZmlsbF9ncmFkaWVudChsb3c9ImdyZXk5MCIsIGhpZ2g9InJlZCIpICsNCiAgICBzY2FsZV95X2Rpc2NyZXRlKGxpbWl0cz1yZXYsIGJyZWFrcz1zZXEoMCwgY2xhc3NlcywgMTApKSArDQogICAgc2NhbGVfeF9kaXNjcmV0ZShicmVha3M9c2VxKDAsIGNsYXNzZXMsIDEwKSkgKw0KICAgIGxhYnMoeD0iY2xhc3MgMiIsIHk9ImNsYXNzIDEiLCB0aXRsZT1wYXN0ZSgiUGFpcndpc2UgcHJvYmFiaWxpdGllcyAtIGNsYXNzICIsIGNscykpICsNCiAgICB0aGVtZV9idygpDQogIA0KICBwcmludChwbG90X2NscykNCn0NCmBgYA0KDQpEaWZmZXJlbmNlIGJldHdlZW4gdGhlc2UgdHdvIExEQSB0cmFpbmluZyBtZXRob2RvbG9naWVzIGFyZSBub3Qgd2VsbCB2aXNpYmxlLCBzbyB3ZSB3aWxsIHBsb3QganVzdCB0aGUgZGlmZmVyZW5jZXMuDQoNCmBgYHtyfQ0KZm9yIChjbHMgaW4gMTpjbGFzc2VzKQ0Kew0KICBjdXJfY2xhc3NfUnMgPC0gZGZfYWdncl9Sc19kaWZmICU+JSBmaWx0ZXIoY2xhc3MgPT0gY2xzKQ0KICBwbG90X2NscyA8LSAgZ2dwbG90KGN1cl9jbGFzc19ScywgYWVzKHggPSBjbGFzczIsIHkgPSBjbGFzczEpKSArIA0KICAgIGdlb21fcmFzdGVyKGFlcyhmaWxsPXZhbF9taW5fdHJhaW4pKSArIA0KICAgIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdz0iYmx1ZSIsIGhpZ2g9InJlZCIsIG1pZD0id2hpdGUiLCBtaWRwb2ludD0wKSArDQogICAgc2NhbGVfeV9kaXNjcmV0ZShsaW1pdHM9cmV2LCBicmVha3M9c2VxKDAsIGNsYXNzZXMsIDEwKSkgKw0KICAgIHNjYWxlX3hfZGlzY3JldGUoYnJlYWtzPXNlcSgwLCBjbGFzc2VzLCAxMCkpICsNCiAgICBsYWJzKHg9ImNsYXNzIDIiLCB5PSJjbGFzcyAxIiwgdGl0bGU9cGFzdGUoIlBhaXJ3aXNlIHByb2JhYmlsaXRpZXMgZGlmZmVyZW5jZXMgLSBjbGFzcyAiLCBjbHMpKSArDQogICAgdGhlbWVfYncoKQ0KICANCiAgcHJpbnQocGxvdF9jbHMpDQp9DQpgYGANCg0KDQoNCmBgYHtyfQ0KbGRhX2NvZWZzIDwtIGxvYWRfbGRhX2NvZWZzKGJhc2VfZGlyLCByZXBscywgZm9sZHMpDQpgYGANCmBgYHtyfQ0KZm9yIChjbDEgaW4gMToxOSkNCnsNCiAgZm9yIChjbDIgaW4gKGNsMSArIDEpOjIwKQ0KICB7DQogICAgY3VyX3BsdCA8LSBsZGFfY29lZnMgJT4lIGZpbHRlcihjbGFzczEgPT0gY2wxICYgY2xhc3MyID09IGNsMikgJT4lIGdncGxvdCgpICsgZ2VvbV9ib3hwbG90KGFlcyh4PWNvZWZmaWNpZW50LCB5PXZhbHVlKSkgKw0KICAgICAgZmFjZXRfd3JhcCh+dHJhaW5fdHlwZSkgKyBnZ3RpdGxlKHBhc3RlKCJDb2VmZmljaWVudHMgZm9yIGNsYXNzIiwgY2wxLCAidnMiLCBjbDIpKQ0KICAgIHByaW50KGN1cl9wbHQpDQogIH0NCn0NCmBgYA0KQ29lZmZpY2llbnRzIG9mIExEQSB0cmFpbmVkIG9uIHZhbGlkYXRpb24gc2V0IGhhdmUgbG93ZXIgdmFyaWFuY2UgYW5kIG1vcmUgc2ltaWxhciB2YWx1ZXMgYWNyb3NzIHRoZSBuZXR3b3Jrcy4NCg0KDQpgYGB7cn0NCmF2Z19sZGFfY29lZnMgPC0gbGRhX2NvZWZzICU+JSBmaWx0ZXIoY29lZmZpY2llbnQgIT0gImludGVyYyIpICU+JSBncm91cF9ieShjbGFzczEsIGNsYXNzMiwgcHJlY2lzaW9uLCB0cmFpbl90eXBlLCBjb2VmZmljaWVudCkgJT4lIHN1bW1hcmlzZSggdmFsdWUgPSBtZWFuKHZhbHVlKSkgJT4lIHVuZ3JvdXAoKQ0KYXZnX2xkYV9jb2Vmc192dCA8LSBhdmdfbGRhX2NvZWZzICU+JSBmaWx0ZXIodHJhaW5fdHlwZT09InZhbF90cmFpbmluZyIpDQphdmdfbGRhX2NvZWZzX3R0IDwtIGF2Z19sZGFfY29lZnMgJT4lIGZpbHRlcih0cmFpbl90eXBlPT0idHJhaW5fdHJhaW5pbmciKQ0KYXZnX2xkYV9jb2Vmc192dCR2YWx1ZSA8LSBhdmdfbGRhX2NvZWZzX3Z0JHZhbHVlIC0gbWluKGF2Z19sZGFfY29lZnNfdnQkdmFsdWUpDQphdmdfbGRhX2NvZWZzX3Z0JHZhbHVlIDwtIGF2Z19sZGFfY29lZnNfdnQkdmFsdWUgLyBtYXgoYXZnX2xkYV9jb2Vmc192dCR2YWx1ZSkNCmF2Z19sZGFfY29lZnNfdHQkdmFsdWUgPC0gYXZnX2xkYV9jb2Vmc190dCR2YWx1ZSAtIG1pbihhdmdfbGRhX2NvZWZzX3R0JHZhbHVlKQ0KYXZnX2xkYV9jb2Vmc190dCR2YWx1ZSA8LSBhdmdfbGRhX2NvZWZzX3R0JHZhbHVlIC8gbWF4KGF2Z19sZGFfY29lZnNfdHQkdmFsdWUpDQphdmdfbGRhX2NvZWZzIDwtIHJiaW5kKGF2Z19sZGFfY29lZnNfdnQsIGF2Z19sZGFfY29lZnNfdHQpDQphdmdfbGRhX2NfdyA8LSBwaXZvdF93aWRlcihhdmdfbGRhX2NvZWZzLCBuYW1lc19mcm9tID0gY29lZmZpY2llbnQsIHZhbHVlc19mcm9tID0gdmFsdWUpDQphdmdfbGRhX2Nfd1ssIGMoImNsYXNzMSIsICJjbGFzczIiKV0gPC0gbGFwcGx5KGF2Z19sZGFfY193WywgYygiY2xhc3MxIiwgImNsYXNzMiIpXSwgYXMuZmFjdG9yKQ0KYGBgDQoNCmBgYHtyfQ0KcmFzdGVyX3Bsb3QgPC0gZ2dwbG90KGF2Z19sZGFfY193KSArIA0KICBnZW9tX3RpbGUoYWVzKHg9Y2xhc3MyLCB5PWNsYXNzMSwgZmlsbD1yZ2IoZGVuc2VuZXQxMjEsIHJlc25ldDM0LCB4Y2VwdGlvbikpKSArDQogIHNjYWxlX3lfZGlzY3JldGUobGltaXRzPXJldiwgYnJlYWtzPXNlcSgwLGNsYXNzZXMsIDEwKSkgKyBzY2FsZV94X2Rpc2NyZXRlKGJyZWFrcz1zZXEoMCxjbGFzc2VzLCAxMCkpICsgc2NhbGVfZmlsbF9pZGVudGl0eSgpICsgZmFjZXRfd3JhcCh+dHJhaW5fdHlwZSkNCnJhc3Rlcl9wbG90DQpgYGANCkZvciB2YWxpZGF0aW9uIHNldCB0cmFpbmVkIExEQXMsIHJlZCAtIGRlbnNlbmV0IHNlZW1zIHRvIGJlIGRvbWluYW50LiBPbiB0aGUgb3RoZXIgaGFuZCBmb3IgdHJhaW4gc2V0IHRyYWluZWQgTERBcyBibHVlIC0geGNlcHRpb24gaGFzIGhpZ2hlciB2YWx1ZXMuDQoNCmBgYHtyfQ0KYXZnX2xkYV9jb2VmcyA8LSBsZGFfY29lZnMgJT4lIGZpbHRlcihjb2VmZmljaWVudCAhPSAiaW50ZXJjIikgJT4lIGdyb3VwX2J5KGNsYXNzMSwgY2xhc3MyLCBwcmVjaXNpb24sIHRyYWluX3R5cGUsIGNvZWZmaWNpZW50KSAlPiUgc3VtbWFyaXNlKCB2YWx1ZSA9IG1lYW4odmFsdWUpKSAlPiUgdW5ncm91cCgpDQphdmdfbGRhX2NvZWZzX3Z0IDwtIGF2Z19sZGFfY29lZnMgJT4lIGZpbHRlcih0cmFpbl90eXBlPT0idmFsX3RyYWluaW5nIikNCmF2Z19sZGFfY29lZnNfdHQgPC0gYXZnX2xkYV9jb2VmcyAlPiUgZmlsdGVyKHRyYWluX3R5cGU9PSJ0cmFpbl90cmFpbmluZyIpDQphdmdfbGRhX2NvZWZzX3Z0JHZhbHVlIDwtIGF2Z19sZGFfY29lZnNfdnQkdmFsdWUgLSBtaW4oYXZnX2xkYV9jb2Vmc192dCR2YWx1ZSkNCmF2Z19sZGFfY29lZnNfdnQkdmFsdWUgPC0gYXZnX2xkYV9jb2Vmc192dCR2YWx1ZSAvIG1heChhdmdfbGRhX2NvZWZzX3Z0JHZhbHVlKQ0KYXZnX2xkYV9jb2Vmc190dCR2YWx1ZSA8LSBhdmdfbGRhX2NvZWZzX3R0JHZhbHVlIC0gbWluKGF2Z19sZGFfY29lZnNfdHQkdmFsdWUpDQphdmdfbGRhX2NvZWZzX3R0JHZhbHVlIDwtIGF2Z19sZGFfY29lZnNfdHQkdmFsdWUgLyBtYXgoYXZnX2xkYV9jb2Vmc190dCR2YWx1ZSkNCmF2Z19sZGFfY29lZnMgPC0gcmJpbmQoYXZnX2xkYV9jb2Vmc192dCwgYXZnX2xkYV9jb2Vmc190dCkNCmF2Z19sZGFfY193IDwtIHBpdm90X3dpZGVyKGF2Z19sZGFfY29lZnMsIG5hbWVzX2Zyb20gPSBjb2VmZmljaWVudCwgdmFsdWVzX2Zyb20gPSB2YWx1ZSkNCmF2Z19sZGFfY193WywgYygiY2xhc3MxIiwgImNsYXNzMiIpXSA8LSBsYXBwbHkoYXZnX2xkYV9jX3dbLCBjKCJjbGFzczEiLCAiY2xhc3MyIildLCBhcy5mYWN0b3IpDQphdmdfbGRhX2NfdyR0b3BfbmV0IDwtIGZhY3RvcihjKCJkZW5zZW5ldDEyMSIsICJyZXNuZXQzNCIsICJ4Y2VwdGlvbiIpW21heC5jb2woYXMubWF0cml4KGF2Z19sZGFfY193WywgYygiZGVuc2VuZXQxMjEiLCAicmVzbmV0MzQiLCAieGNlcHRpb24iKV0pKV0pDQpgYGANCg0KYGBge3J9DQpyYXN0ZXJfcGxvdCA8LSBnZ3Bsb3QoYXZnX2xkYV9jX3cpICsgDQogIGdlb21fdGlsZShhZXMoeD1jbGFzczIsIHk9Y2xhc3MxLCBmaWxsPXJnYihkZW5zZW5ldDEyMSwgcmVzbmV0MzQsIHhjZXB0aW9uKSkpICsNCiAgc2NhbGVfeV9kaXNjcmV0ZShsaW1pdHM9cmV2LCBicmVha3M9c2VxKDAsY2xhc3NlcywgMTApKSArIHNjYWxlX3hfZGlzY3JldGUoYnJlYWtzPXNlcSgwLGNsYXNzZXMsIDEwKSkgKyBzY2FsZV9maWxsX2lkZW50aXR5KCkgKyBmYWNldF93cmFwKH50cmFpbl90eXBlKQ0KcmFzdGVyX3Bsb3QNCmBgYA0KDQpgYGB7cn0NCmNvZWZzX2dyaWQgPC0gZ2dwbG90KGF2Z19sZGFfY193LCBhZXMoeD1jbGFzczIsIHk9Y2xhc3MxLCBmaWxsPXRvcF9uZXQpKSArIA0KICBnZW9tX3Jhc3RlcigpICsgDQogIHNjYWxlX2ZpbGxfYnJld2VyKHR5cGU9InF1YWwiKSArDQogIGZhY2V0X3dyYXAofnRyYWluX3R5cGUpICsNCiAgc2NhbGVfeV9kaXNjcmV0ZShicmVha3M9c2VxKDAsIGNsYXNzZXMsIDEwKSwgbGltaXRzPXJldikgKw0KICBzY2FsZV94X2Rpc2NyZXRlKGJyZWFrcz1zZXEoMCwgY2xhc3NlcywgMTApKSArDQogIGd1aWRlcyhmaWxsPWd1aWRlX2xlZ2VuZCh0aXRsZT0iTmV0d29yayIpKSArDQogIHhsYWIoIkNsYXNzIikgKyANCiAgeWxhYigiQ2xhc3MiKSArDQogIGdndGl0bGUoIk5ldHdvcmsgd2l0aCBoaWdoZXN0IGxkYSB3ZWlnaHQgZm9yIGNsYXNzIHBhaXJzIikgKw0KICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSwNCiAgICAgICAgYXhpcy50aWNrcyA9IGVsZW1lbnRfYmxhbmsoKSwNCiAgICAgICAgcGFuZWwuZ3JpZC5tYWpvciA9IGVsZW1lbnRfYmxhbmsoKSwgDQogICAgICAgIHBhbmVsLmdyaWQubWlub3IgPSBlbGVtZW50X2JsYW5rKCkpDQoNCmNvZWZzX2dyaWQNCmBgYA0KDQpMREFzIHRyYWluZWQgb24gbm4gdHJhaW4gc2V0IHNlZW1zIHRvIGJlIGRvbWluYXRlZCBieSB4Y2VwdGlvbi4gTERBcyB0cmFpbmVkIG9uIHZhbGlkYXRpb24gc2V0IGJ5IGRlbnNlbmV0Lg0K